import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from python_ai.common.xcommon import sep
pd.set_option('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', 1000, 'display.expand_frame_repr', False)
plt.rcParams['font.sans-serif'] = ['Simhei']
plt.rcParams['axes.unicode_minus'] = False

spr = 2
spc = 4
spn = 0
plt.figure(figsize=[16, 9])

sep('data')
df = pd.read_csv(r'../../../../../large_data/ML2/train_titanic.csv')
print(df[:5])

sep('check nan')
print(df.columns[df.isnull().sum() > 0])
print(type(df.isnull()))
print(df.isnull().sum())

sep('PassengerId')
df_id = df['PassengerId']
print(df_id.value_counts().value_counts())
del df['PassengerId']

sep('Pclass')
cnt_df = df.Pclass.value_counts().sort_index()
print(cnt_df)

idx_s = df['Survived'] == 1
idx_d = np.invert(idx_s)
dead_cnt_df = df['Pclass'][idx_d].value_counts().sort_index()
print('dead')
print(dead_cnt_df)
sur_cnt_df = df['Pclass'][idx_s].value_counts().sort_index()
print('survive')
print(sur_cnt_df)
print('survive rate')
print((sur_cnt_df / cnt_df).sort_index())

df2 = pd.DataFrame({'s': sur_cnt_df, 'd': dead_cnt_df})
spn += 1
ax = plt.subplot(spr, spc, spn)
df2.plot(kind='bar', ax=ax)

sep('Names analysis')
import re
regex = re.compile(r'(.*, )|(\..*)')
def x_filter_name(name):
    return re.sub(regex, '', name)
df['title'] = df['Name'].map(x_filter_name)
print(df['title'].value_counts())

sep('Emerge title')
sep('Mlle => Miss')
idx_Mlle = df['title'] == 'Mlle'
print(df[idx_Mlle])
df.loc[idx_Mlle, 'title'] = 'Miss'
print(df[idx_Mlle])
sep('Ms => Miss')
idx_Ms = df['title'] == 'Ms'
print(df[idx_Ms])
df.loc[idx_Ms, 'title'] = 'Miss'
print(df[idx_Ms])
sep('Other => Rare')
xcommon = ['Mr', 'Miss', 'Mrs', 'Master']
df.loc[~df['title'].isin(xcommon), 'title'] = 'Rare'
print(df['title'].value_counts())

sep('Analysis by sex')
sur_cnt_sex = df['Sex'][idx_s].value_counts().sort_index()
dead_cnt_sex = df['Sex'][idx_d].value_counts().sort_index()
df3 = pd.DataFrame({'s': sur_cnt_sex, 'd':dead_cnt_sex})
spn += 1
ax = plt.subplot(spr, spc, spn)
df3.plot(kind='bar', ax=ax)

sep('Analysis by title')
sur_cnt_title = df['title'][df['Survived'] == 1].value_counts()
dead_cnt_title = df['title'][df['Survived'] == 0].value_counts()
df4 = pd.DataFrame({'s': sur_cnt_title, 'd': dead_cnt_title})
spn += 1
ax = plt.subplot(spr, spc, spn)
df4.plot(kind='bar', ax=ax)

sep('fillna for Fare')
print(df['Fare'].describe())
sep()
# df.loc[df['Fare'].isnull(), 'Fare'] = df['Fare'].mean()
df['Fare'].fillna(df['Fare'].mean(), inplace=True)
print(df['Fare'].describe())

sep('fillna for Embarked')
print(df['Embarked'].describe())
sep()
df.loc[df['Embarked'].isna(), 'Embarked'] = 'S'
print(df['Embarked'].describe())
spn += 1
ax = plt.subplot(spr, spc, spn)
ax.set_title('Embarked')
sns.countplot(data=df, x='Embarked', hue='Survived',
              hue_order=[1, 0],
              ax=ax)

sep('drop ')
# del df['Cabin']
print(df[:5])
print(df.info())

sep('Hypothesis of age')
x_train = df.loc[df['Age'].notna(), ['Survived', 'Sex', 'Pclass', 'Fare']]
x_test = df.loc[df['Age'].isna(), ['Survived', 'Sex', 'Pclass', 'Fare']]
from sklearn.preprocessing import StandardScaler, OneHotEncoder
onehot = OneHotEncoder()
x_oh_train = onehot.fit_transform(x_train[['Survived', 'Sex', 'Pclass']]).toarray()
x_oh_test = onehot.fit_transform(x_test[['Survived', 'Sex', 'Pclass']]).toarray()
std = StandardScaler()
x_std_train = std.fit_transform(x_train[['Fare']])  # ATTENTION DOUBLE
x_std_test = std.fit_transform(x_test[['Fare']])  # ATTENTION DOUBLE

x_train = np.c_[x_oh_train, x_std_train]
x_test = np.c_[x_oh_test, x_std_test]
print(x_train[:5])
y_train = df.loc[df['Age'].notna(), 'Age']

from sklearn.linear_model import LinearRegression
model = LinearRegression()
model.fit(x_train, y_train)
h_test = model.predict(x_test)
df.loc[df['Age'].isna(), 'Age'] = h_test
print(df.info())

sep('Family analysis')
print(df['SibSp'].value_counts())
print(df['Parch'].value_counts())
df['family_count'] = df['SibSp'] + df['Parch'] + 1
print(df['family_count'].value_counts())


def x_group_family_cond(cnt):
    if cnt == 1:
        return 'small'
    elif cnt <= 3:
        return 'mid'
    else:
        return 'large'


df['family_type'] = df['family_count'].map(x_group_family_cond)
print(df['family_type'].value_counts())

sep('Analyse family')
spn += 1
ax = plt.subplot(spr, spc, spn)
sns.countplot(data=df, x='family_type', hue='Survived',
              order=['small', 'mid', 'large'],
              hue_order=[1, 0],
              ax=ax)

sep('Analyse Fare')
spn += 1
ax = plt.subplot(spr, spc, spn)
df['Fare'].plot(kind='kde', ax=ax)

sep('Analyse Cabin')
idx_cabin_na = df['Cabin'].isna()
df['Cabin'].fillna('_n', inplace=True)
print(df['Cabin'].value_counts())


def x_process_cabin(x):
    return str(x)[0]


df['cabin_type'] = df['Cabin'].map(x_process_cabin)
print(df['cabin_type'].value_counts())
df.loc[df['cabin_type'].isin(['G', 'T']), 'cabin_type'] = 'O'
sep('G/T => O')
print(df['cabin_type'].value_counts())
spn += 1
ax = plt.subplot(spr, spc, spn)
# df.sort_index(axis=1, inplace=True)  # ATTENTION This no use at all!
sns.countplot(data=df, x='cabin_type', hue='Survived',
              hue_order=[1, 0],
              ax=ax)

plt.show()
